{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3b69b129",
   "metadata": {},
   "source": [
    "# 生成式预训练语言模型：理论与实战\n",
    "深蓝学院 课程 \n",
    "课程链接：https://www.shenlanxueyuan.com/course/620\n",
    "\n",
    "作者 **黄佳**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a168a35",
   "metadata": {},
   "source": [
    "## 第1步：构建实验语料库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d3216379",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "词汇表： ['Niuzong', 'Student', 'Teacher', 'Kage', 'Boss', 'is', 'Mazong', 'Xiaobing', 'Xiaoxue']\n",
      "词汇到索引的字典： {'Niuzong': 0, 'Student': 1, 'Teacher': 2, 'Kage': 3, 'Boss': 4, 'is': 5, 'Mazong': 6, 'Xiaobing': 7, 'Xiaoxue': 8}\n",
      "索引到词汇的字典： {0: 'Niuzong', 1: 'Student', 2: 'Teacher', 3: 'Kage', 4: 'Boss', 5: 'is', 6: 'Mazong', 7: 'Xiaobing', 8: 'Xiaoxue'}\n",
      "词汇表大小： 9\n"
     ]
    }
   ],
   "source": [
    "# 定义一个句子列表，后面会用这些句子来训练CBOW和Skip-Gram模型\n",
    "sentences = [\"Kage is Teacher\", \"Mazong is Boss\", \"Niuzong is Boss\",\n",
    "             \"Xiaobing is Student\", \"Xiaoxue is Student\",]\n",
    "# 将所有句子连接在一起，然后用空格分隔成词汇\n",
    "words = ' '.join(sentences).split()\n",
    "# 构建词汇表，去除重复的词\n",
    "word_list = list(set(words))\n",
    "# 创建一个字典，将每个词汇映射到一个唯一的索引\n",
    "word_to_idx = {word: idx for idx, word in enumerate(word_list)}\n",
    "# 创建一个字典，将每个索引映射到对应的词汇\n",
    "idx_to_word = {idx: word for idx, word in enumerate(word_list)}\n",
    "voc_size = len(word_list) # 计算词汇表的大小\n",
    "print(\"词汇表：\", word_list) # 输出词汇表\n",
    "print(\"词汇到索引的字典：\", word_to_idx) # 输出词汇到索引的字典\n",
    "print(\"索引到词汇的字典：\", idx_to_word) # 输出索引到词汇的字典\n",
    "print(\"词汇表大小：\", voc_size) # 输出词汇表大小"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d564e419",
   "metadata": {},
   "source": [
    "## 第2步：生成Skip-Gram训练数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "de8f4ae7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Skip-Gram数据样例（未编码）： [('is', 'Kage'), ('Teacher', 'Kage'), ('Kage', 'is'), ('Teacher', 'is'), ('Kage', 'Teacher'), ('is', 'Teacher'), ('is', 'Mazong'), ('Boss', 'Mazong'), ('Mazong', 'is'), ('Boss', 'is'), ('Mazong', 'Boss'), ('is', 'Boss'), ('is', 'Niuzong'), ('Boss', 'Niuzong'), ('Niuzong', 'is'), ('Boss', 'is'), ('Niuzong', 'Boss'), ('is', 'Boss'), ('is', 'Xiaobing'), ('Student', 'Xiaobing'), ('Xiaobing', 'is'), ('Student', 'is'), ('Xiaobing', 'Student'), ('is', 'Student'), ('is', 'Xiaoxue'), ('Student', 'Xiaoxue'), ('Xiaoxue', 'is'), ('Student', 'is'), ('Xiaoxue', 'Student'), ('is', 'Student')]\n"
     ]
    }
   ],
   "source": [
    "# 生成Skip-Gram训练数据\n",
    "def create_skipgram_dataset(sentences, window_size=2):\n",
    "    data = []\n",
    "    for sentence in sentences:\n",
    "        sentence = sentence.split()  # 将句子分割成单词列表\n",
    "        for idx, word in enumerate(sentence):  # 遍历单词及其索引\n",
    "            # 获取相邻的单词，将当前单词前后各N个单词作为相邻单词\n",
    "            for neighbor in sentence[max(idx - window_size, 0): \n",
    "                        min(idx + window_size + 1, len(sentence))]:\n",
    "                if neighbor != word:  # 排除当前单词本身\n",
    "                    # 将相邻单词与当前单词作为一组训练数据\n",
    "                    data.append((neighbor, word))\n",
    "    return data\n",
    "# 使用函数创建Skip-Gram训练数据\n",
    "skipgram_data = create_skipgram_dataset(sentences)\n",
    "# 打印未编码的Skip-Gram数据样例\n",
    "print(\"Skip-Gram数据样例（未编码）：\", skipgram_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b81e0a8",
   "metadata": {},
   "source": [
    "## 第3步：对Skip-Gram数据进行One-Hot编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "72158e33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "One-Hot编码前的单词： Teacher\n",
      "One-Hot编码后的向量： tensor([0., 0., 1., 0., 0., 0., 0., 0., 0.])\n",
      "Skip-Gram数据样例（已编码）： [(tensor([0., 0., 0., 0., 0., 1., 0., 0., 0.]), 3), (tensor([0., 0., 1., 0., 0., 0., 0., 0., 0.]), 3), (tensor([0., 0., 0., 1., 0., 0., 0., 0., 0.]), 5)]\n"
     ]
    }
   ],
   "source": [
    "# 定义One-Hot编码函数\n",
    "import torch # 导入torch库\n",
    "def one_hot_encoding(word, word_to_idx):\n",
    "    # 创建一个全为0的张量，长度与词汇表大小相同\n",
    "    tensor = torch.zeros(len(word_to_idx))  \n",
    "    tensor[word_to_idx[word]] = 1  # 将对应词汇的索引位置置为1\n",
    "    return tensor  # 返回生成的One-Hot向量\n",
    "\n",
    "# 展示One-Hot编码前后的数据\n",
    "word_example = \"Teacher\"\n",
    "print(\"One-Hot编码前的单词：\", word_example)\n",
    "print(\"One-Hot编码后的向量：\", one_hot_encoding(word_example, word_to_idx))\n",
    "\n",
    "# 展示编码后的Skip-Gram数据样例\n",
    "print(\"Skip-Gram数据样例（已编码）：\", [(one_hot_encoding(context, word_to_idx), \n",
    "          word_to_idx[target]) for context, target in skipgram_data[:3]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03235c50",
   "metadata": {},
   "source": [
    "## 第4步：定义Skip-Gram模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "31ae5270",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Skip-Gram模型： SkipGram(\n",
      "  (input_to_hidden): Linear(in_features=9, out_features=2, bias=False)\n",
      "  (hidden_to_output): Linear(in_features=2, out_features=9, bias=False)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# 定义Skip-Gram模型\n",
    "import torch.nn as nn # 导入neural network\n",
    "class SkipGram(nn.Module):\n",
    "    def __init__(self, voc_size, embedding_size):\n",
    "        super(SkipGram, self).__init__()\n",
    "        # 从词汇表大小到嵌入大小的线性层（权重矩阵）\n",
    "        self.input_to_hidden = nn.Linear(voc_size, embedding_size, bias=False)  \n",
    "        # 从嵌入大小到词汇表大小的线性层（权重矩阵）\n",
    "        self.hidden_to_output = nn.Linear(embedding_size, voc_size, bias=False)  \n",
    "\n",
    "    def forward(self, X): # X : [batch_size, voc_size]        \n",
    "         # 生成隐藏层：[batch_size, embedding_size]\n",
    "        hidden_layer = self.input_to_hidden(X) \n",
    "        # 生成输出层：[batch_size, voc_size]\n",
    "        output_layer = self.hidden_to_output(hidden_layer)  \n",
    "        return output_layer\n",
    "    \n",
    "embedding_size = 2 # 设定嵌入层的大小，这里选择2是为了方便展示\n",
    "skipgram_model = SkipGram(voc_size,embedding_size)  # 实例化SkipGram模型\n",
    "print(\"Skip-Gram模型：\", skipgram_model)"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAARwAAAGDCAIAAAB2pjghAAAgAElEQVR4nOydd3hURffH5/bdzWaTQBqQhBICAYL0LggoItIkYv3hC4ooKL6IIghYQMFeUMCC8CJN1ACvFBEREaQGDSUQCCWBQCCkkLK72d3bZn5/HNl3DQLJ7oa0+Tw8PLs3986de/d+75k5c+YMQwhBFArFf7BVXQEKpbZBRUWh+BkqKgrFz1BRUSh+hoqKQvEzVFQUip+hoqJQ/AwVFYXiZ6ioKBQ/Q0VFofgZKioKxc9QUVEofoaKikLxM1RUFIqfuaWiqug0E4xxJdWEcuupO79mpYgKY6xruq7p8FVVVUKIrukMw8AWu91OCAGNee7mPlyWZYQQwQQOhP8dDoeiKO793R/c+/ir/lAUIQRjjDF2V4wQ4nK5PC/TvZt7h3+sBpTjuQXKhEu43iGKoiiKcr0dnE4nfHDfE/j/piV74r6HXmO32z2/yrIMt8jlcsHZVVWFX1PXdYQQ/IieH1wuFzwh8BkOgT/dQIfuy/e8RqfTiTH23EKu4uNlVgi+MgplWfZ0xun9+/c/+sijuXm5q1evHjhwYEJCAkJIUZRjx46dOXPm/sT7f9/1+7Zt2zDGJpOJENKyZcu+fftGREQcOHDg3Nlzg4cMDgwM1DVd07U1362JiYnheX7Hjh02m41lWZfLFRAQIMvy0KFDu3frznKsW7G+A0URQgoKCpYvX15UVOR0OoODg8PDw/v06dO6devLly9/8803JSUlLpfLYrFomtaxY8chQ4bAgTabbevWrXv37s3Ly4uNjb333nvbtm1rMBjc5RcUFPz444+DBw/meX7hwoVWq1VRlPDw8MLCQovF8sADD8THxyuKcvz48fXr12dlZZlMpnvuuWfwvYNzLuesWrWquLhYVVWTyeR0OlmWjY+Pf+SRR9ZvWI8QSkxMFAQhOzt706ZNKSkpqqq2b9/+kUceCa0fyvHchQsXvvnmm4EDB7Zr1w4h5HK5Dh8+fPr06X/9619e3yuTyYQQcjgcv/76a1JSEsuyDRo0GDlyZKdOnVRVzcnJWbNmzejRoyVJgv1TU1NPnjz5r3/9a+XKlWfPnsUYWywWh8NBCBEEYezYsfv3709PT2cYRpZlo9HYqlWrwYMH8xz/9bKvmzdvfscddyCEFEWx2+1LliwZNmzY/v37U1NTQUuyLBsMhoYNGw4ePLhhw4aiKHr/EPhApYgKIZSTk7Nr165Ro0Y5HI69e/ZmZWW98sor4eHhDMOcO3fu8OHDQ4YMyc/PLygoSExMDAoKKioq2rlz5+7du2fNmpWVlbU/eX//O/sHBgZyPCcr8r59+3iev/3223v16iWK4n//+19BEO6++26WZWNiYjiek2XZ/bP5C5ZlbTbb0aNHhwwZEhoayrJsZmbm22+//fLLL7Msm5KSct9994WHh/M8n5GRsXLlypycnLFPjOV4bsGCBSdOnBg/fnyjRo2Sk5OXLFnSv3//kfeP5HgOSrbZbNu3b+/SpYvRaITyw8PDBUFgGAZjHBERYbfbf/jhhzVr1jz++ONjx449c+bMqlWrsrOzJ0yYcPvtt2OM09LSdu3a9fDDDwcHBwcHByOEzp8/r2kaQujs2bPTp09v1arV+PHjRVHcuXPnv//976lTp7Zv195ut+/Zs+f8+fMzZ85s2LChJEkFBQXp6emEEK9fSbIsE0I++eST9PT0xx57rFmzZocOHXrvvfeGDh06fPhwhmHS0tJKSkqCg4I5jkMIXbhw4ejRoyUlJd26dWvXrl1+fv7q1at79uyZkJBgNBpFUTx06FBoaGivXr1kWS4oKPjuu++KiooefPDBzMzMAFOA+7z5+fmHDx++8847W7duHRoaihD6/PPP+/fvHxcXJ4piWFiYH1+yFaVSRKWqKsMwHMcxDKNpWqAlMDc3d9WqVRMnThQEQRRFg8FgMplEUZQk6e6771YURZKkjh07Tps2LScnx2g0IoREUVRVVRAEjLHRaJQkqVGjRo0aNUIIbd++nWXZLl26cCynqIqu6aIo6prufmp9BFodLMvC09a+ffvY2FiWZbt3756dnb1v376+ffuyLJuQkNCsWTNJkjp16hQSEvLzzz8XFhXu3bv33Llzb775ZuPGjTHG4eHhYWFhy5Ytu/322xs2bAiykSRJ13Vo4QQEBHTu3LlJkyYcxxFCRFGUZfnChQu///77pEmT+vXrhxCKiYkRBGH9+vXnzp3r1asX1C01NbVLly6RkZFgzA0GQ0lJCcb4u+++69ix46RJkyRJwhi3bdtW1/W1a9c2a9aM47h69erBkzp58mSGYRiGASuBrtrnimI0Gn/55Zfz58/PmTMnOjpaUZTY2Nj69esvWbKkffv2QUFBBoOhtLQUEyywAiGEZVmO40wmU0KbBI7nXC7X9u3bY2JiwATBJTRv3rxTp04sw7IcK8vy/v37hwwZommawfiXtRdFEV55BoOhadOmCKFLly6Fh4cnJCT07t0b9qnCPBGV0qcSBIHjOGhDw+cnn3zy0KFDe/fuVRTF4XDYbDan06koSkBAANYxGJlLly6BxSeEqKoqiiLcFyhKVVVVVaGHYzKZFEURRZHjOYPBwHIsQgj+9wssy7pbgLIscxzHsixCSNf13NxcODvIAyEEO2RlZRkMhqCgoDNnzrRs2bJ+/fq6prMsazAY2rdvbzQajx079leZmJSWljIMA40TuHZ4i4uiCJI7ceKEw+Fo0qQJ1Mdms/Xu3fv999+HB0hVVbgnUCDDMgzDWK1WXddzcnLOnj3bu3dvnuPR1ffCXXfdlZ2dXVBQwHEcx3Hjxo07fPjwnj17CCGaphmNRoK9f/4URfnjjz9iY2OjoqLQ1VdhQkJCvXr10tPTMcbQXIcLZBhG13VN0xwOB7wBS0pKSktLAwICEEJ/vRx13Wq1CoLA8RzDMPBiFUWR4zir1eo+r9FoNBqN0LNSVdVoNGqaBrYaVamiUOU1/9zY7XaTydSkSZM+ffp8++23cXFxkiSBYCRJOnfu3NfLvrbb7cXFxZmZmR06dAgLCyspKREEgWVZnucRQvBM65qu6zp4BcAGVnbNAZZlk5KSAgMDbTbbuXPndF3v06ePoihWq3Xx4sWiKMIjYrVax48fTwi5cOFCu3btzGaz2wdgMplCQkIuXLgAXzH5W+c7Pz9/ypQpoCWXyxUfHz9jxgxd18PCwiwWC+xjMBhAABhjnuM5jsMYC4IA94dlWUEQzGaz3W6Hzkb9+vVZjgXTTQgJCgpSFKW0tJTjuICAgMaNG/fu3Xv58uXNmzfneZ7jOEwwQxivb2l+fn6HDh0IIVjHHM8JglAvpJ7JZLLZbAzDwCngrQRIkuRuq3McBzsghDie4zDH8/wff/zhdDqdTmdpaekff/zx0ksvWQItCCFd1zHGLMtijEtLSwkhcAfgqXA7JKo8l1GliyowMBBeMPfdd19aWtq6deuio6PhLsMLjGXZRo0aNWzYMDExsW3bthhjg8EAr1iCCWL/cuBwPLxnObizt+zGwZuyXki9oKCgnj17du3aVZKkU6dOBQUF9erVKyIiYvv27Xa7fc6cOU2bNnW5XNDaQVctJ9bB84fdjgr3tcNzFhgY+Prrr0dHRwcFBV25csVsNkuSFBQUpGkamHqEkCAIuqYLguCuFcMwLpcLfGWKouh/p7i4GOv4LwPOsjabTRRFk8nE87z7t0hJSfn+++9jYmLgVnt9fwRBMBgMLpdL0zT4mRBCmq6BJ4kQ4v69QLSyLLuvHV31i7rFAJjN5tDQUGj5P/3002D24T0CB7IsC10GMHHiVQRBqHJFoVsgKkVReJ5nWbZ+/fpPPvnk+++/f/HiRZfLBS/7yMjIRx55BNoMkiQRQhRFadCgAXxACBk4A3iWJUnieZ5hGJZlVVXVNA1aQZVtskRRHD58eNOmTXVddzqd8IoFhbRq1SouLq5nj57vvvfuJ5988uqrr9avX79ly5ZHjx4FdSGENF2zWq35+fnx8fE//PBDw4YNu3btyl8FIcSyrKZpYEzq168Pz0RMTExubm5xcXFISIggCIqiZGRkzJ8//6WXXoIWINwcTxiGMRgMISEhDRo0OHHiREJCgslkgmqcO3fOYrE0aNDg4sWL8GIKDQ0dPXr0/Pnz4bdA3nao4MDY2NjTp09rmgaeQEVRSkpKrly5Eh0drWmachVBEAgm4PWGAQOGYdx/hQOBbl27JSYmui0tODkDAgKgywA3pKioyOVywXsH/Pjut0yVU1mDv+4+CUJIVVWe5zVNa9GixZ133nnw4EGEkCAIoBO73Q4dDNhZ4IUGDRpcvnx569atsiwzDLNly5bCwsL4+Hj3SwhjDK8oggnjgR/rD61znucDAgIcDgfDMPBKRghBQxQugWM5jufGjBlTVFS0cuVKWZYHDRpks9m++eYb+IGtVusHH3wQGRnZsmXL7Ozs7du322y2zMxM6G5BsweatdDFglGm8PDwzp07r1y50m63W61WjPGqVassFktERARcPvRU/+pTMQwYLmj4DR48ePPmzQcOHFBVVeCFo0eP/vDDD3379oW3uCzLPM/LstytW7c777xz3759sMWXu3f33Xfb7fbVq1dDD8dqtX700UeRkZFdu3SNjo6OjY1dtmxZfn4+QujkqZN79+5t164dz/PQjIdGPrRvJUkyGU0Mw2i6hhDSdA3GNqGbfdttt+3evTslJQUudtu2bREREfXq1UNXHzZ4Kfv0q/uJSrFUiqIQQjiOgx6k0WiUZRlaLyNGjEhLSwNrA04weL254XiuefPmY8aMWbJkybx58wwGg9lsHjduXEREBJh+l8slSZKmaZVq63me1zUdnBDw6DscDhAVy7FmszkgIEDTNIZlMMYR4RFPPfXUBx980LVr106dOs2YMWPBggVPPvkkx3ElJSU9evQYO3ZsYGDgwIED33jjjXHjxhFCEhMTo6KisrKyigqLZs6cqaqq2WyGWzd27Nhhw4aNGjVqyZIlT459MsAcUFhYGBcXN3nyZPCLsiwLjtMyFTabzaqq3nXXXYSQJUuWLF26FAz+E088cffddyOEJEmC3wKOTUxMTEtLkyTJs8PjBTExMZMmTfryyy8feeSRBg0a5OXm9e3Xd8iQISzHcjw3evToefPmPfHEE3B1d95556BBg6BpB71B8FJCf8zpcoJbAiEkiiJ4g3RdNxgMAwYMSE9PnzVrFrxqExISnn32WbgQcPDAEHN1gPH7cwnNYnAtMCxDMIGRWU+XNzRLwINXpgLuljfG+OLFi6qqNmjQICAgwN2jAB8gIcRzONXvQFfK/cH91f1XlmE5ntM13T3uDNcLDyjEXthsNqPRGBgY6D7Q6XRmZWU1aNAgKCgILtOtDafTyfM8nAXuDEKooKCguLjYaDTCWMKNcd9h6GVdunTJbDbXq1cPygSB8RwPjmxQJpwXtOr1jdJ1HXzcxcXFeXl50dHRYFuQR6syMzNT07SIiAi4cHCculwudxvV8yoQQgz714Hu+wnef0VR8vLygoKCIiIiyvwowPWe51s5bOV/UQE2m83zYXI/mu6WBgwawt101wGcE/CYyrIsy3JgYOC1t8Ptc6+MmsNLgWEY6Ovf4Mf4X3QSJjCiwnGcLMtmsxn6BgEBAdBtKPP4wvViHcOjA80/hBDHcjrWNU2TRAlODR0PELDT6TQYDHDhUKb7tYUQ0rEOw8foqibhdHAt4IP9a2BA08GHATuDxfD6XqGbPa+e5cPlwJPgdgl6Dj17bi9zeElJCfQ8EULwLgY7Bq1f9261VlTuMm9wJdee98aX7f79fBn+94KbPjde1Mfz2it0bJnKuPVfppyKDubeylta0WuHul3vaXH/tQrjJ66lcqPUK6Nt6d8CfeeWvgKv8cf45ezV6om8lhv86Ne+UKoDdD4VheJnqKgoFD9TWY4KCqXOQi0VheJnqKgoFD9DRUWh+BkqKgrFz1BRUSh+hoqKQvEzVFQUip+hoqJQ/AwVFYXiZ6ioKBQ/Q0VFofgZKioKxc9QUVEofoaKikLxM1RUFIqfqfRkmhQKLEoEiVkghah3QIIXyLbrv9r5HyoqSqWzfPnynJwcWIcKkih6jaqqDodj6tSp1VlXVFSUSsfhcIwZMyYsNMz3tfkURVm2bFl1y/RSBtqnotwKVFXleA4WMfAaaPupqlpYWFjVF3QjqKWiVDqQSR8++GJk4FjI++23ylUC1FJRKH6GiopC8TNUVBSKn6GiolD8DBUVheJnqKgoFD9DRUWh+BkqKgrFz1BRUSh+hoqKQvEzVFQUip+hoqJQ/AwVFYXiZ6ioKBQ/Q0VFqSx0Ta/qKvwzlb0kL51PRfE/hBCsYx3rWMU8zyuKgjFGVydE+f1EHM8pioIQEkURIYQxhnwYeXl5u3fvvuuuu/Ly8mRZbtOmDfyVYRhZlhFCKSkpP//8c0BAQPfu3Xv17FXqKN23b1+7du0iIyN9qRW1VBT/wzAMx3OCIAiCwDAMy7JgHFRV9eNZCCEwoRghJIoiKIoQQjBxuVyqqi5dujQgIIDn+SVLlqSnp0MdQH6SJG3dunXRokVt27aNjY1dtGjRj5t/NJvNJpNp7dq1PpoyKipKpaCqKsMw8HR+9913xcXFhBBBEPx4isLCwj179qiqqqrqgQMHDhw4oKpqaWnpzt93FhcXHz161OFwdO7c+ezZs/n5+UeOHElLS7PZbBzHMQxjs9lOnTo1bNiw+++/f8SIESNGjDhw4ADLsk2bNj137lxaWpovFaOiolQKoB+bzTZq1KhDhw7FxMT4ve2Xn5//3//+98qVKxjj77///vvvv8cYFxUVJSUlKYqya9euZs2a1a9fX1EUq9VaWFioqmpgYCCkczIZTS+88EJiYiLU6uzZszzP65oeFRUVFxe3e/du91m8sFpUVBR/YrPZoPuEECoqKhowYMA333wTGRmpKAq01vx1Ioxxw4YNQ0JCUlNTL126VFpampOTk5WVdfLkyQYNGlgslrNnzzZs2BAh1KFDh4SEhO7du3fo0AEhBN0thJDT6UQIqaq6Zs2a48ePP/XUUxzP6ZoeHR19/vx5q9XqlpNn8pny1I06Kih+Q9f0wMBAePIuXLgwYsSIlJQUhFCDBg0QQoQQH5P+ecIwjMViiY2NPXHihCiKDRo0MJlM0HJLSEiw2+2lpaVmsxkhBG3C4ODg/Pz8DRs25OXlsSw7ePDgli1b2my2b7/9Njk5edKkSQ0bNiSE6FgPCwuzWq3uExFCKmpjqagofoPjOXgEjxw5MnTo0AsXLsD22NhY2O7HFiA4/W677bZly5a5XK527dpZrdbk5OTCwsLhw4ezbNkEgxjjwMDApk2bhoaGMgwTHh6uadr8+fOvXLkyc+bMxo0b+6tiiIqK4iNlXuSyLB8+fHjQoEHFxcUGg8HlcqGrlsq/fSpN1ziei4qK0jQtNTV1xIgRVqt1w4YNUVFR4eHhhJCgoCC73Y4Q4jguICDA6XQKvNC/f39FUQRBcLlcSUlJGRkZY8eO5Xk+OzvbaDSGhYVxLJefn2+xWHypMBUVxVdUVRUEAbpSmzdvHjVqlNPpFAQBnNcmk6lhw4bunoy/kCTJ6XTWq1evTZs2Bw8eDAsLkyQpICCgefPm0OpLSEhIT0/v3bu3wWAICQlJSkqSJGnYsGGiKBJCHA7Htm3bsrKyZs+efeXKlZCQkFatWn380ccIoczMzObNmweYAuBE3kjLl4yhFArGWNd1+Dx//nyDwQDPFcdx4ADs2rXrZ599dvHiRdjZ9zN++OGH+fn5hBBFUTRVczgc7mJdLhdUxmaznThx4uWXXwZXvqZqZ86cKSgoKFMBp9OpqZrL5dJUDaqXk5Mzffr0c+fOua+uDOWpIfX+UXwCxnYxxsuWLXvuuefc6WMJIWCdbr/99so4r67pgiDoWDcajW5jIkkSnNRsNrdo0SIqKmrnzp0YY47nYmNj69evDzqBcAqEEDhOoMJOp5NhmB2/7WjWrFl4eDjyIZqJioriPZ6P3UMPPbRgwYLGjRvLsgxNLHjWW7duXRnrCYBTpMzCPG5vPkKIYZgHHnigfv36ICFZluGv8JUQIsuyrusM+1fIktFodLlcTZs1HTVqlHvY2juoqCjeAwtGgX4kSZowYUJqauqAAQPcjTGEULdu3Tz3r9T6QDwUgBAKCwvr2bOnwWAghIiiCGc3GAwQ0CSKoiRJDMOIoshyLCFEkqSuXbtKkuRjrnYqKopPeOoE2oEnT55cv379/fffjxAKCQlp3ry5370U1Zy6dbWUyubdd9/t169f//7916xZs3PnzsTExGqlqFuzsBV1qVN8gniMU509e3bhwoWZmZmSJKmq2qdPnz59+qBb9ShXH6ioKD7BMAzWsaZrkiQ9//zzr7/+emhoqMvlcvvWb319quS8nlBRUXyCYRgd66qq7tix48SJEytWrNA1HTwBdRYqKopP6JrOMIzZbJ45c+bbb79tNpurVSeqSqjr10/xEYZlBEH44osvwsLCwC1RbVNT3DKopaL4BMuyJSUl77zzzvr1690DRJAHwr2P59hRdejzVDbUUlF85Y033hg2bFirVq3gK8PWftncGGqpKD5x5MiRpUuXHj58GEKTqro61QJqqSgVxjPE7tVXX502bVpMTIx/MyXVaKilolQYgomOdUzwjh07zpw5k5SUhDEWBMHTUnn2nSRJghBB3x2DuqZzHOeell89PY1UVJQKw3IswzAMZmbMmPHGG28wDKMoyg1Ge0tLS3/88cfQ0FBZln3JUiaKIsuy+fn5RqORVDx1xC2DiopSYRiGcblcq1evDgwMHDlyJMZY07QbPOXDhw8vLCzkOM7pdJaZrFEhjEajoigtWrSoqnCNcuLTvBFKncJTNnl5eW3btt2zZ0/z5s1vGpQE8+3hcF+eN03TYN5+9Wz1uaGiopQLcE7AhCWs48kvTNZ1ff78+dX8+a4SqKgoN8etKFVVNU3Lysrq0aPH6dOnQ0NDq7pq1RHap6LcHDBHkBaC5/lp06ZNnTo1ICCgqutVTaGiopQXjudcLte+ffuysrJWrVoFSR2quc+gSqCiopQXGN595plnFi5cCE48qqh/hPYyKeUCPHjLly9v0qRJz549OY6jvfHrQR0VlHKha7qsyC1btly/fn3Hjh3BB8iwDPX+XQu9I5SbAK9dTddef/31e+65p2PHjgghhmEYltH1uj516h+hlopyXTwnQZ04caJ///5Hjx4NCwsrM4xbbcOFqgoqKsp1wRirqspzPEJo5AMj+/TpM3nyZPT3xLRUUddCvX+UGwG5Wrdt23bs2LFly5ZVdXVqBrRPRbkuBP/V/Hvuuec++ugjnuchBXlV16u6Qy0V5bpwPIcQmj9/flxc3JAhQ2CjF+nHduzYkZ6ezrJsmTlXFQUmZfE8/6/H/gV1q55QUVGuCyGksLDwtdde81yt3QtOnDjRtGnTli1bmkwmX0TlcDgMBsPKlSvtpfagoCBfqlSpUFFRrgvW8ezZs0eNGhUbG+tLORzHtWzZsmnTpj7WhxCiaZrFYqnmg2NUVJS/4Tlp6tTpUytWrDh27BisRuN1mYIgmEwm5OGj9w5oQFqtVqfTGRgY6HU5lQ0VFeVvwJJTmqbxPD916tQ5c+ZERkSWyeNXUVwuF8QN+r5APcbYZDJV87TSVFSUsui6LsvyL7/8kp6e/v3331d1dWoeVFSUv4ExZhk2MDDw1Vdf/eCDD9yr6NIggfJTrTt8lFsPwzAsxy5cuDA0NHT48OGqqnpm+aOUB2qpKH+DYRir1Tpr1qzt27dDNr9q7mqrhtD7RSnLjBkzRo8e3aJFC3R1Ij1t+1UIaqkofwMWbjt37px7hXaqqIpCLRXlb7wy85V33nknJCQEvlJFeQEVFQURQmw2G0Lop59+Opd1bsyYMe4/MTej6mpdfaGioiCGYSCFyyuvvDJr1iyO4+hqiL5A+1SUvxL6zZs3LywsbOjQoQghWZZhFYKqrlqNhIqKgjiey83Nfeedd3bt2gVbBEGgivIa2vyro8iy7F6mTdf0OXPmPPLII3FxcQghvywk5YnZbPZjaeUH5lN6+lp0Tbfb7dfb3+Vy/WMhxIPynJdaqjoKxliSJMg0ln4yfc2aNenp6ZV3OpZl3QnZK+8sZYBRAU8lcDxXZoEsVVVhCTmWZd11c2fedd+lCp2XWqo6itFoVFWVYRhM8OTJk2fOnAnT/ipjMTWbzQZx7pqm+bfkGwOWJzs7OykpyeVyXbhwISsryz3+BkDIiKZpGzduzM7ORgjJsrxjx47MzEzkbVobKqq6iyAIiqLs2LEjLy/v6aefrrxFe/Pz8xVFYRjGj1M2wO7d2IYYDAan07l69eqoqChREJctW5aSknLtbk6n8+uvv/7qq68KCwsRQpIkBQQE/Pzzz/B+8SL0kYqqjuJwOBBCsixPmDDhiy++4Li/2kW+myl3z8ThcHzzzTcdO3YsKCjwu/XTdR0Udfbs2R07dpSUlKiqmpKScvDgQV3TMcbbtm27fPny0aNHCwoKbrvtttSjqVlZWUeOHElPT4cD4SXidDpfeumlY8eOhYaGghHTNb1Xr17Z2dluBVZ0XI6Kqo4iiRJCaMmSJQkJCV27dvW9q+Pux0uS5HA4Pv/883bt2o0ZM8Zms7Vu3fra7o2PsAwLlsRkMq1du9ZqtWKMly5dumTJEofTkZOTs27dOk3TUlNT4+PjAwICnE5ncXGxoih2u11RFBhFQAipqjp27Nhp06aVWRkoNjb20KFD3vlsqKOijsLx3JUrV959992tW7f6xXnAMIzT6VRV9cMPP/zPf/6TnZ0Nzol33323uLjYx4n018LxnKqqLMtaLJagoKC0tDRN02RZ1nU9KysrMzMzMDBQFMXjx4/fe++9DMP07Nnzt99+69ixY+fOndHVVwDDMGazuUOHDpcvXy5Tvejo6M2bN7tcLqPRWNG6UVHVUVwu1+epxUAAACAASURBVNy5cx999NG2bdv6Uo6nY2PatGmLFy/WNE3XdVBUu3btBg0a9M0331RGDCGUaTQaE9okpKamiqIYHR2NMc7JyTl16lS7du1MJpPVagX3ncvlKioqkmXZZrOtWLHC6XSKotinT5/4+HhRFAMCAsr4/SVJcrlcVFSU8oIxPn36dFJSUmpqqq7pvuTQU1W1oKDgzTff/Prrrz3Hecxms6IoH3zwgRcPZTnhWA4hRAhp1brV/uT9Bw4caNWqFSxLV1RUdOedd/I8bzKZJEnSNE2SJIPBoKoqIaR169Y2m43neaPRKIoiwzCyLLtcLng7wN0AlzpsqWiaayqqugI0eKClx7Ls9OnTn3/++aCgIIJ9siEY4549e+bk5JSxRYqidOvW7a677vrHEVXfIYSwHAsnbdy4MUIoOTl50KBBCKGkpKTY2NjWrVszDCNJUlFREfg5wX5aLJa+ffu6Kw8rLZhMJlVVbTYbDFuxLJufnw8mTlVVWOGu/FBHRV2BYRiCCcZY1/RNmzZlZmY+/fTTuu6TmUIIsSy7ZcuWgICAMh55RVE++ugjhJDBYKjsqPbg4OAmTZo0atSoWbNmkZGRUVFRCQkJkiRxLNe2bduMjAwQdkxMzOrVq9etW4euGh+WZSHqwuFwcBwnSRLP85Ds+ty5cz169DCZTDA0XDEIpY5hs9latWq1c+dOyKLsOy6X6+DBg2X6JMOHDyeEaKrmcrm++uqr8+fP67oOfnBf0HV9/vz5RUVF2ANN1RRFgdPpuq4oCnxVFOXUqVNTpkw5d+4cOEsyMjIKCgr+sWT33bDZbPn5+S+++CJ4LzRV8zxXeSpJLVWdY+nSpdHR0X369PHabiiKAkOiMENEkiTQDEIIuigIobfeekvXdFVToRFFCCGY+NjUvB4sx8LiCRBZz/P8X18ZNiYmpnXr1idPnrRYLISQpk2b1qtXz/NYtxLQVfNlNpv37NnTrl07yNeJyf8Gf8t5x2ifqm6Rm5v7yiuvJCcn+1IIz/PQN+N4jhCydu3aSZMmbdmyxWq1Dh8+nGXZcePGxcfHsyyLCYZgKFh50U8XUV44nlM19aGHHjp8+HB59of4CZfLZTKZBg4cCL4KL5p/1FLVIVRVnT179pNPPuljbnTPca2VK1dOmjRp27Ztffr0GXTPoNWrV5tMpilTpoDhgmA/QRAgxO7WJ2aCQeeePXuWZ2cY+DKZTAMGDIBM17AFDFT5DTu1VHWI1NTUjRs3njx50sdUfhhjkMcXX3zx3nvv/frrr/Hx8Q6Hw2QyjRw5slOnTlFRURCvAGG7wK2PUkcIQchFOXfmOM5dSUVRRFEUBMEL7x8VVR1iypQp06ZNMxqN3vWmyNVxXhjYffPNN1euXPn7779HRUXJsgzPLsuybjMIXRSe56sqoQUpx4jztbWCLe7YX47jKlpzKqpajs1mgw73li1bHA7HM88847WiVFWFR40QMnXq1K1bt+7evTsiIkLX9BuHn9eULDH/WEMa+0cpC8SJYoyfffbZxYsXwyPibr+VH5i4Icsyy7ITJ048ffr0b7/9FhwULMvyTefee3rYqr+0fIeKqvbjcrk+/fTTJk2a9OvXz+l0chzHsZzXLqpx48adP38+KSkpJCSEZdnqvExoVUFFVcuBPvebb76ZnJwMk8NhOylfPBvxiJctLi4eM2YMxnj79u032O3GxZbzvDUa6lKv5WCMZ86cOWbMmPj4+IoeSwgpLS2Fz4qi3HfffUajccOGDdfb2aeK1iKoparlHDlyZNWqVbA4fEWfe5huRAjJy8tLTEzs2rXrxx9/7EUd6lqfilqqWs6rr7763nvvhYeHe11CXl7eXXfd1aVLlw8//JCao/JARVU7gRnj69evP3/+/NixY2FMs/z50AkhuqYrinL27Nk+ffo8+uij8+bNc8cWXEuN8JjfMqioaiEYY57nWY6dPn36hx9+yDBM+R3okBAGwosyMzN79uw5fvz4qS9NrcTq1jpon6oWAnbj448/jomJGTBgAEzFK6clgYm6giCkpaX17t17wYIFDz/8MLVCFYKKqhbicrmsVutrr70GSbZgFlM5c+4xDKNr+t59ex944IHPFn6WeH8iXZ60olBR1R5kWYZhKKPROHHixKeffrpFixaEEJ7nsX7zCFp3mMX237aPHDlyxYoVw4YN80vFIDssIUSWZU+jV1EDCFdnt9s1TavOxpOKqjYADmtJknRN17GelZW1devW06dPu3eAdA7X80m4txNCNmzYMHHixE2bNvXu3dtf1Wvbtu3FixdPnz5dphVaUWHA4eHh4SHBIf6qW2XAUCdpLcPpdN5///1DhgyZMGGCqqqe+fhv/BBjjJcsWfLee++tWLGiU6dOEF3ur1qBGbyesCtaFLrlU0gqBLVUtQR4alVV/eWXX86ePTthwgRUblMAxy5cuPCTTz7ZsGFD69at/V49ggliEdax5/zfigpM0zSY7Ojv2vkZaqlqCbqmsxzrcrkSEhIWL158xx13wMN3vVg7twjBIr300kubN2/esGFDdHS0H5cRqJtQS1VLgJwnn332Wft27fv16wepyDDB15u1CpKDxuH06dN37dq1ffv24OBgqijfoZaq9nDp0qVu3bpt3ry5bdu27mXLbhwVjjEeP358SkrK1q1b69evf0urW3uhlqrGA71/RVFmz5792GOPtWnThhAi8IJ7zPcf35vg4B43btzly5d/+uknqig/QkVV4wH7c+bMmR9//PHUqVOqqkqSxPEciAp5GCj3SBTG2Gq1jn1irKqpW37a4s6fjOpGFHllU90dKZRyMnHixOnTpwuC4J6GeK2XTNM0EI8syw8++CAv8Bs2bKBTd/0OFVUNxuFw6JququrGjRttNtvo0aMFQbhB+jGe5xVFcblc99xzT4sWLb777jtIMUvxL1RUNRiO43Sscxw3ZcqU999/H7KZ33gYp6SkpFevXh07dnz77bcxxoqq3KrK1iFuhajcad39VaB7gYnKW/vZ70DKfOSPOoN5sdlskiSJovjuu++2atXKvTzMtcDkKIRQVlZWjx49EhMT3333XbPZzDAM5GH1sT6UMlS6S51cXeL7+PHj/hoL53m+adOm5Y+8rg7Isux0Oq9cuYIx9lpXhBCWZSVJ0nXdYDBER0dfvHixQ4cOO3bsuGkYxKlTpwYNGvTCCy88++yzdru9zBK3bqijwncq3fsH3t6dO3f++eefPqbwdnPkyJEhQ4Z07NjRL6XdGniOnz9/frt27TDGMBHQOyAvOczJnTZt2htvvPH44483b978BocQQtLS0gYNGvTyyy8/++yzCKHyZ0KmeEGliwrsiaZpAwYM8JcMgoODISjbL6XdGjie4zhu4MCBsLSM1+XAuJOqqvPnzz9+/PimTZv++OMPT4tdJmgV3mhPP/30Z599NnToUOQR2+rL5VBuwK2wVDClR9d1fzUtMMa6rnuRZrVq4TiO53jkWxPLbrdD4vzS0tKJEye+8sorDRs2LLNur6qqLMPCwkpbt259+umnV65c2a9fP3c+I6qoSoUO/tYwAgICGIax2+2HDh1yOp1PjXuqjKIYhgHHusAK69evf+qpp9avX9+9e3eIuK3CmtcdqKhqGGBnRFFMSUlZtWrV9YZuWZZdsWLFlClTfvjhh/bt28M6N9RA3RqoqGoeDMNomjZ+/PguXbq4A2fL8NVXX82dO3f//v1NmzaFLVRRtwwqqhoJy7KiKLIsC3M3wDkBAtM1/eN5H3/xxRd79+6NjIykWrr1UFHVYNxr2Hj+P/etucuXL9+9ezdVVFVBRVWDKeNFFAThueeeO3LkyO+//x4ZGYkQul7jkFKpUFHVEhRFeeKJJ/Ly8rZs2QLBR7Isi0KNiTipTdwKUbkXffDXOBXkMaYBNXBLdU1nWOb//u//XC7Xhg0boK+FEKI2qqqgAxc1GEKI1Wp1upz33nuvKIpr1qwxGAw1a0C8VkJ/gBoMy7I8zw8cODAqKmr5suUgJ0WhszmqGCqqGkxubu6AAQO6dOmyaNEiGAWWZZlj6UzeKoaKqoYBXnKWZYuKivr373/nnXfOmzdPVVUIsBRFkcYiVTnU+1fDUBSF53hRFLdu3frcc8+Ne3IcBOzTIanqA32r1TAgU5KiKI8++ujo0aNZjpVluaorRfkbNVJUN1hXszpDCMEE+2hSIK+LoigOh8NoNMqyXIOmP9cRKiAq9zqwLpcLXR19UhRlx44dhw4d0jU9JSXFarUqirJq1ars7OxKqzNiWbZaOY4///zzWbNmpaWlFRcX65peZsI8ZGD217nc6Zp1Xdc13T0qRak+XPf3cD8HLpcLPjMMw7CMKIowqqgoiqIohYWFW7dujYmJuZRz6dNPPz1//rwoiuHh4d9++63L5aqklol/08j4Tk5OzuzZs7t27dqtW7fXZ72ekZHB87zT6cQYK4rCsizHc8QDf52XYZlrVyGgVDnXFZXbiXTmzJmUgym6pttsts2bN1+8eBEhlJubm5KSgjHeuHFjVFSUxWI5dOiQ3Wb/448/CgoKOnfuXFBQcObMmUqa8a5pmqqqIHW1GuB0OhFCDofj3Llzc+fOjY+Pv+222z777LOLFy+61wdgGIbjOJ7n/WhYqKKqJ9f9gaHHomv6mTNnfv3111JHaU5Ozpdffvnrr78ihA4cOLBt27YrV66kpqb269dPEIT8/HxMcG5uLs/zwcHBUVFRO3furKRKw1x6HeuEEFwNAF82x3FglzDGp0+fnjZtWkxMTOfOnT/++ONLly6hq1kA/KsBqqhqyI1c6hhjhNBtt922Y8cOm8124sSJsLCwU6dOYYyPHz/epUsXh8OhKIrFYiGE9O/ff//+/Q899BBkZYmNjd21a5fdbocMj/5C1/Q/U/4cOnSoKIrXWyTm1gPZkURRBLUjhNzt3kOHDh06dOiFF154++23VVUVBMG9Mi+ltnKj55JhGIZj6tWrJwjC6dOnz54927Vr17S0tDNnzly+fHnEiBH5+fk8z1ssljKOOIZhQkJCiouLZVn2l6ggbJTjuc6dOm/cuNFkMvXt27ea9NFfeeWVuXPnKorizqliMBgge+aQIUMefvjhYcOGmc1m0BVVVK3nJqJCCAUFBXXq2Gnv3r1XrlwZNWpUdnb25s2bOY6Ljo7Oz8+/VfVEEIaDMeZ4zmAwYIyhoXXLKnADNE2DyhBCYP3MPn36jBo1asiQISEhIeiq4VIUBVL2uScXUmolNxKVu73eJqHNmrVrmjdvHhsb26hRoy1btgwbNsxoNIaFhWmaZrVay5gjQkhRUVFwcLAf38o2my0wMBCWjuV5HkquJpYKRG40GgcPHnzvvfeOHDnSYDCUWcFa13SDwSDLMsbY80/eQbtS1ZmbP5QMwzRs2LBevXrg5YuPjxdFsWvXrgihBg0aNGjQ4OLFiwzDhIaGsiz72muvHT16FOv44MGDUVFRfuxQmc1mQgispATuAfjgr/J94bbbbvvvf/9bUFCwYsWKxx9/PDAw0C0b8KHDV03TEEJGo9H3MxJCdu/evW/fPoZhsI6p3atW3Lz5hxCqX7/+okWL4Dnu16+fOy2jwWBo37793r17b7vttoCAgE8++SQjIyM6Ojq/IF9RlMGDB9+KK6gG3HfffZqmiaJ4Kw1IkyZNhg4dunbt2jvuuAPTpLPViXI1n3RNh9T4bqcWwzDgy+rRo0dpaWl2djYsIdGmTRuLxbJ///64uLiQkJA6EpYGuY1QuVPP+vj0w2/RtGnTNWvWPProoz/++GM1aQZTgHL9GKqmwjoD0JOBRhf8HxER8cADD1itVkiRhRByOByRkZFDhgyxWCx158eGu3HL1lCDifQdO3ZcvXr1E088kZSUhOiCHdWGcg31wMwCzxRzkCQVniRYcsI9AcFoNHbr1g22cJzfJsx5PjHVMKAWXh/X5ot1V5Jh/1dnf9XcYrH07t178+bNAwcOLC4uHjduXLW6J3UWn8ZPb/oT0t+4smEYpnXr1jt37kxMTGRZduzYsXRwucqpLkEJFC8A/RiNxjZt2mzfvv2OO+7Izs5+/fXXcU1bD6WWQW99DcZtkXRNb9So0Z49ezZu3Pjvf/+7amtFKZeoGB+o7AuoKXjeEF9uC8y4gcFltxdR0zWMcURExPr165OTk59//nm15qyGXPuglqrmIUmSw+EoLS11j1jwHM+yrKIoDRo02LZt26lTpx577DGqq6qCiqqGIYqirumSKFksFgh6Qle9jjALWNf17777DmNcdwbfqxtUVDUPTdc4nvvhhx80TSP4rxYgxLsghIKDg4OCgr7//vuGDRv279+/SmtaR6GiqmFgjCVJcrlcISEhU6dOdY+MlenBYoy//vrrDh06dOrUCSYTQIoRt/YqY3o/BaCiqmGAaeI4rn379tu2bTt+/DhsZ/6+Ij241D/88MPhw4fffvvtubm5DMNwPAcLO1AhVSpUVDUVnucnT5784osvon8KJrTb7QghjPFrr702fvz4Pn36nD9/3v1X9/7UQ1sZUFHVSEAVTz/99OnTp7dv315GGKIoSpLktlcTJ06cPHly586d//zzz78Ox9RSVSKVHlHhnpXgx1YHwzCyLOuafr212ashkHcJ4ifB2c0y/3ujMWx5zQXHcxBIAUt+vPPOOy+++CLkXfS8G+4JXbqmC4Lwf//3f8FBwSNHjly1alXPnj1r0H2riVS6pVJVFQYrIYbdL2iaFhAQwPHcLYsK9x3IVONyuRRF4TiO4ziWY//3r9xAUQ6HA95QI+4bERoaunz5ctAJeCOufXkFBgYOHjJ4/vz5iYmJO3fupH2qSqXSLZU7lOa33347evSoX8osKSkZNmyYoig1K+OxoihfffUVJABEf3ctlD9UT5Kk4uJijLEsy5qmcRw3e/bsBx54YOjQoZAPAxPMMf8zRO78jYGBgUOHDl21atWDDz64ZMmSPn36BAUF+e3aKB7cCkcQtABtNpu/jJUkSRaLRVVVlmFrUEvG4XDAcC3Mqy8zmaWchei6znGcrusmk8lisSCEnE7n6NGje/Xq9e9///sfy3H7zUG6e/fuffTRR+fOnTtkyBCqq8qg0kXlLv+v4ZGrXWSYX+R7+TXIeeXHVY8BdyftyJEj/fr1O378OCxKX+ak7s+Qf4bjufT09KFDhz777LPPP/+8H+tDAW6dqCqJOisqsD+6roOu/v3vf7tcri+//LLMKcqIyuVyQTTT+fPn+/Xr9/zzzz/33HOgNKfT6ZekNBQqqluH3y0VQkjXdEywIAiXL19u3br10aNHw8PDPVOgeQ5JIYRUVdV1XeAFVVPtdvvAgQMHDx78xhtvVFL16iZUVLeOynhqCSFYx6CrBQsW/Prrr+vWrYOzQKyt5xkhY4znV7vdftddd3Xp0mXBggU1a4iiOkMjVmownqN/0BSMj49fsWJFt67dOJ6D9RA8MxD+o6R1TR88ZHCjRo2WLFni3uhyuWDBJIoXUFHVYMr0lxBC367+dtnyZe6kZbDDTc2jqqr3338/IWTt2rXuUQpquLyGhinVKh544IGcnJw1a9bA13KG9hFCfvjhh/r1699zzz1OpxOkqGp0jqOXUFHVElwul6qqiqp8+umnL7/8sqIo5c9kKopiaWnpf/7zny5duvTu3fvKlSvIY9SeUlFo868WMmLEiL59+06aNMmLY6dMmbJ169atW7dGRkZijP2YqBBjTDDBxPv099A/dDtgqq2PioqqFnLixIkBAwYkJyc3atSooscqijJv3ryvvvpq8+bNcXFxyE+dK4irMhqNvjxvmqaBrqp5hBoVVe3kqaeeCgwM/PDDDyt6IEho4cKFn3766aZNm/yiq3Xr1h05csRkMvm4OiusWClJ0gsvvODfFTr9DKHURi5fvmyxWHJycip6oMvlgin3y5cvj4qK2r17NyjBFxYtWpSZmVlaWqrruqZqXv8jhGiqtnDhwoKCAh+rVKlQR0XtJCQkZMaMGTNmzKjogeCf0DV91KhRCxcuvO+++7Zv304IgY2KonhRGUKI0Wg0GAwEE47nvP5HCOF4TlVVP+borwyoqGohGGNRFCdMmPDDDz+kpaWBJMrvDIRsFgihwfcOXrVq1ZgxY3bu3AkLw/q+BmRdgIqqFgILEFsslrlz5z733HPgJauoixyOuvvuu9etW/fII49s2rQJ1aigsCqEiqp2oiiKrulPjXsqNzd306ZNxKulXCHWqWvXrrt27ZoyZcrnn3+O6HLD5YCKqnZiMBigH/LWW2+99tprhBBd9yb1AIQ7NW/efPPmzR999NHbb7/t/hMkSvBbjWsRVFS1nGHDhlkslpUrV3rmmfGC5s2bHzx4cPHixVOnTrXZbLCRWq1/hIqqlsMwzMcffzxz5kx7qd3Hosxm8549e/bu3TtjxgxCCF1d7npQUdVydE3v0KHDnXfeuXTpUl/KkWWZYZjIyMgff/zx+PHjjzzySLUefq1SqKhqOZBN6d13350zZ05xcTFs9GKVHVjQmRASFBS0adOm4uLiIUOGWK1W9E/OeqfT6fkVY+x74F8NgoqqlgOzGCMiIp588smZM2dCpkSe9yk1ndFo3LJli9FoHDFiBEJIEAS3axHKr+MTHKmoajmqqmqapqrqyy+/vG7duozMDOSP4SaMcVJSUps2bTp37myz2cD0KYoCXng/1LsmQ0VVyxEEQRAEnuctFsuUKVNeeeUVdHX2hC+AM/2DDz646667+vbtW1BQoCgKx3Kqqk6bNi01NRV2I1fzDbIsy7CMjx7ImkKduEgKwzAsyz4/6fmjR48mJyd7t3a9e9liQgh0sQRBeOuttx588MFevXplZ2dzPDdp0qRPPvnk1VdfdR8iy/L/klZzVfC8lYnSuvEwgCzLEFLsebgn5Tljpad9plQfNF2bPXv2888/v2/fPn+VybLs5MmTQ0JCevTo8eijjy5atEjX9Y0bN+7Zs6dTp04Gg6EKI5vI1Vw3uqa7vf83rs+1gwRYxxV9F1BLVYdgWXbEiBEMwyxfvtxfZSqKIknS+PHjhw4dOm/ePHiXcxz30ksvgbuCEKJ44K/zlgcwqhAfXB6Hp6ctIoSoqurdRDIqqjoE9K/mzJnjGW3kIzB+9d1338E4mCiKoigSQvbt25eUlIQQkiQJOnVwdn+dtzxcunQpKSlJluWLFy9mZ2e7vSnX23/9+vXZ2dmEEJvN9vPPP1++fJnlWC+GH6io6hz9+/dv27bt7NmzvS6B8SAwMHDjxo2jR4+GP0FAIPRJpk+fDhshah76db7XH4A+knstJV3T3Z9hWMzlciUlJTVq1IgQsmjRouPHj+u6jjHmeT4/P//ixYtglFwuF8wTW7hw4dKlS3NychiGMRlNBoPhxx9/ZBjGi5hJKqq6yOzZsz///PPi4mK/LPBVXFzcokULdHWAmGEYCLbIzMz86KOPEKTRxbj8Hf3yIEnS5cuXf9vxGww0Hzp86NDhQwghh8ORnJycl5eXnp5+4cKFFi1aXLx48eLFi3/88cfJkydB2H/++efixYvRVaeF1WadMWPG6dOnzWYzTH9kObZ9+/ZZWVlpaWlejLlRUdU5CCGtWrUaNmzYm2++KSvlnbl4PZxO52OPPZaamrpu3bpOnTqBeCCZBELo/fffh6iLysBut69evfry5ctWq3XlypUrV660Wq2XL19evny53W7fvXt3u3btgoODCwoKVFUtKChgWVaWZVVV3QoHN6au6w8//PBLL70UHBwMtpRhGIPB0KpVq61bt6K/W+byVIyKqg4BPgOGYZxO59y5c7///vvi4mIfrYfRaISW2PDhw/fs2bNjx4577rmH4ziWZSVJKiwsdCef8XtSscaNGzdr1uzMmTMFBQWlpaWlpaUFBQXHjh2LjIysX7/+2bNnw8LCCCEdOnRo0qTJ8OHD27Rp88N/f3jttdfWrl175syZadOmvfnmm/n5+aGhoZ07dw4KCjKbzbCULkJIkqTIyMjz5897MaZHRVW3EASBEGIwGEJDQydMmDB9+nTfPXLusEBZlu+4446ffvrp2LFjo0ePxhirqvrRRx/l5+f7GBh1LbIsC4IQFxeXlpaWkZERGxsbFhZ24sSJ06dPt2nTBiGEMY6MjHRnk4eF9rp26zpixIju3btHRETcd999AwYMqF+vPtwBl8vlNrCqqsqybDabXS6XF/1AKqo6hKehwDqeMmXKrl27Dh486JfAIoZh3IM8LVq0+M9//nPx4sUZM2YYDIbt27erqkoI8eMazXCuli1b5ubmpqSktGrVKj4+PiUl5cKFCwkJCSzLlpaWgrsfkvU6HA6WZZs1a9a5c+cWLVpYLJbu3bp37979BuUzDOPdnaGiqqPA8MusWbOmTZtWUlLie4G6psuy7Hb9QbzFnDlzMjIygoKCCgsLGYbRsT8XPieExMXFYYwPHTrUvHnzuLi4Q4cO8TzfpEkTSZLgpCzLCoIAKQfR1ZVyLRZLs2bNdKzruv6PK3qyDMswTF5eXkREhBfNYyqqOgosbfrwww8XFxf//vvvvhsrlmMlSYJlGhFCdrs9KCgIYyxJ0vDhwwMDA9HVvM1+Qdd0rGOTyRQeHh4YGAiWKjAwMDY2lmVZjuOaNm2al5eHEGJZNjIy8uuvv/7mm28IJhjj9u3bP/bYY6IoQn3+MSJRVdWzZ882btzYm36gtwkDKTUbWL1KU7V9+/Y1b94cNvqeN/Mf+fLLL2FcyNPt5gVw+Lx584qKiqD++BpgT1mWMzMzX3jhhaKiIjgwIyMjNzdXlmX46sm1V63r+vnz51999dULFy54UU9qqeoof3mIWaZL5y5xcXFffPEFQgh8X278flKs+3NWyLU2hFxNGsXzfGRkZJs2bX799VfYMzo6OjQ0FDKwlznw2nIYhjl48GCTJk2ioqK8qBgVVd3Frav333//7bffLigocLlcVV0pn4BQeOg4GY3G9GOcFQAAGQ9JREFUxMTE0NBQQoiu6YIguP145O+L5f2jqIxG49ChQxFCXtwTKqo6DTxS8S3j+/Xr9+mnn/7ji7wGQa6uHAkfAgICevfuzTCMpmuQCBF2K88FDhgwIDg4GCHkxfIiVFR1HYZhMMGvv/76559/funSJfB9V+fVn26MpxVyWydBEERR9Iw3v2mEBMMwf7kx6DgVxQsIIVFRUc8+++zcuXNFUfRvz6dKKKMWPwbylgcqKspfi6m9/PLLmzZtSk5OrpL5ubUJevsoyGQyYYwNBsMLL7zw+uuvQyRBZXj/6ghUVBSErjaQJk+efOXKlZ9++ukWt5f8SHXoDdbUe0epJGbOnDl16lSMMUSgUryAioryNwYNGhQUFLRs2TK6vpvXUFFR/gbDMO+///7cuXM9JxfS/lWFoKKi/A1RFHv06NG+ffv33nsPXU2TQkVVIaioKGVxOp1vv/324sWLs7OzvYgnoFBRUcpiNBqbNW02ZsyYefPmwZaa6wysEujNovwNXdPtdjvHc5MnT16+fHlaWlpV16jmQUVF+Rscz0GCsYiIiJdffnnWrFm+z4GHNJqE+Gd9KlVVq/lSPVRUlOsyYcKEgwcP/pny57XLulUUVVUhRNWLOX9uYFSX48qVw7kKoQsUUK6L0Wh86623xo8ff+DAAV/KcTgc+/bti4yMhMx7XpcjCEJpaWlJSQmkOvKlSpUKtVSUG/Hggw+yLLthwwZfCrnnnnvMZrPD4fDRwmiaZjab7777bsh4UW1h6BAE5cb88ssvzzzzzLFjxyArmLsZRrke1FJRboTNZhswYEB8fPwnn3zicrmcTidV1E2hlopyI+DxyMzM7NGjx5kzZwIDA7GOvViyqU5BLRXlRsBMimbNmt17771vvfUWLKBW1ZWq7lBLRSkXBQUFrVu33rt3b3R09LVreFI8oZaKUi4CAgImTpw4a9YsqqibQi0VpbwUFBS0b99+7dq13bp1q+q6VGuoqCgVYMmSJWvXrt28eXNVV6RaQ5t/lPKiqurYsWOvXLmyadMm+i6+AdRSUcoFjPk6nc5ffvllxowZhw4dovPtrwe1VJQKIArisGHDwsPDv/zyS4SQX1aLq31QS0WpAGCvjh07NmDAgNTU1LCwsKquUXWEWipKBYAYpVbxrQYNGvTZZ5+V/0B/vbtrRJZPaqko3nDp0qXWrVsfO3asPCs4HTt27NSpU9AH8+V5g0lZkiQNGDCgOifPoPOpKN4QGBg4fvz4WbNmffnFlzcNXNq9e3d0dHRkZKTL5TIajV6fVBRFu93+22+/derUKTIy0utyKhsqKkqFwRgHBARMnz69RYsW48eP79y5800Pad++fb169XxRFLq6DHZycjLHVev4Q9qnolQYXdcZhgkKCpo+ffqbb76JEFJV9QYTEKEj5KOiahBUVJQKIwgCeCzGjx+fkZGRnJwMqV2qul7VBSoqijfASrgGg2HWrFnjxo1DNIutB1RUFG8wGAyKomCMR44caTabFy9e7DkjuI4LjIqK4iWiKELm2g8//HDOnDnIQ0uQUawqK1elUFFRfKVNmza33377e++9B8bK5XKBJ6Oq61VlUFFRfMVsNr/22mvz58/Pz88Hr3cdd1pQUVF8QlVVlmWbN28+bNiwefPmsSxbJtahDrYDqagoPsEyLMhmwYIFixYtyszMRAhB+nXYXiZ1c9XW9tZARUXxCY7nGIYBj8XEiRNnzpypqiosREAIwRgTXCeE5AkVFcU/MAzz4osv7t69OyUlRRAEp9PJsizLshzPwdBwdVg3/tZARUXxA9CuM5lM77/3/vTp0xFCRqPxwIEDI0eO1DVd13VN06pD2+/W1IGKiuI97mcUZtqzLPvwIw9fuXJl8eLFDzzwwO23375u3boT6ScYhvF7h8put3t+9Qw+dC+ode2CxZqmufdx7wYLkbhcLtgTgkUcDofXdaNR6hTv8WzOQbxsbm5ufHw8BC4BJ06c4DjO7w0/SZJcLpd79TdPJz7Hc7qmczyXn5+/f//+AQMGFBQUCILQqFEjmP+vqqqu6xAUwjCMruu//fbb4cOHMcYdO3a844477Hb7Tz/91KNHj/LMFrsWaqko/kGW5fnz57dt2zYpKQm2QNrNo0ePVsbpeJ43GAyqql670CMhhGEZm82WlJQUFBRkMBhWrlx55MgRXdehSk6n026365ouCALLsFu3bv36668bNmzYtGnTRYsWrVmzRhCEsLCwb7/91ru1f6ioKD4BTSxVVU+cODFz5sz8/HyEkCAIHMdBs2r//v2oEhbgIYTk5ubu3buXYRmEUHJycnJyMkKoqKgoJSUlLy/v8OHD+fn5HTp0OH369Llz51JTUy9fvgxV3bZt24IFC3SsMwzjdDnT09Pvv//+hx56KDExMTExERYN6tix45UrV06dOgUXWCGoqCg+wbEcQojjuPbt26ekpDRr1gxdbV+xLMswTGpqKvRkGIbxfflgNwQTh8OxdOnSCxcuWK3WpKSkZcuWFRcXFxUVffvtt4IgHDhwoHHjxsFBwRhjWZbz8/Nzc3MVRREEISAgwGazQc3NZvPUqVMH3zsYIaQoSm5ublBQEMbYbDa3aNFi69atXszbp30qik/AXHoYp4qOjv7999+7d+9+6dIljDGELBUUFBQWFlbGeSMjI0NDQzMyMmJiYlwuV2lpaX5+fmZmpslkQghdvny5e/fuHM+1bds2JiamW7dunTt3XrFixaFDh0Bgk1+YHBIS8swzz0RERCAWEUI2bNiQnp4+efJkuJyoqKhjx47BVVSoblRUFL8himKDBg1SUlL69et3/PhxdHUCfE5Ozl9rYPt1GR6j0diuXbvjx4/b7fbo6OjS0tKTJ0+mp6e3btVa1/XS0lJBEMBjgRDieV5V1V69erVp0yY5OTkjI2PkyJFgtRBCV65cSUpK2rVr14svvhgbGwvlBwUFlZSUEEwq2p6jzT+K34AmX3h4+K+//tq3b1/YiDEuKiry+7nARdGhQ4fLly8fP368ZcuWcXFxqamp58+f79S5k6qqPM9jjMFjIQgCNPyioqI6duzYsmVLg8GQkJDQqVMns9nsdDo/++yzo0ePzp49u2PHjuiqP13TNFEUvXgRUFFR/IYgCBCaFBkZue2XbcOGDYPt2dnZfnepswyrYz0qKsrhcKSkpMTFxcXFxaX8mSKKYmhoaFBQEMdxxcXFLMsGBgZyHKfrOmQ4QwgFBAQ0btwYOksY4xUrVmRkZDzzzDMmkykzM7OgoMBkMsmyXFJSYjQavRiwos0/ij+BWCRCCMdz33333ejRo5OSki5cuKDrursl5i90XbdYLNChioqKstlsoWGhLVu2NJvNgiDExcWdPXsWfHcRERFr165VVfXhhx9GCHXr1q1Lly6qqhJCYCzr6NGjEyZMYFlWVdW77777lZmv8Byfnp7esmVL6KFVDEKhVAJWqxU+PP7444IgXLx4UVM1H8vUdZ0QMm/evLy8PMjQpOs6bLyWjIyMF198MS8vDyqTnZ1dWFgIO3seouu6pmoYYygKiiWEZGdnv/jii9nZ2fBXN+WpJ23+USqFwMBA+PDVoq8GDx6clpaGrg5q+QVoyBFMyrjmMMYQZ9SsWbN27drt2rULIQThFCEhIbAPBM7DMBqE/DIMAxPD/iqWkD/++KNt27aNGjXSdK2idaPNP0rlwvHcoEGD2rRpw3Isy/ntJU4I+cdVvVmWdccu3XPPPZmZmU6n051y0NMJWWadVfhKrgYKRkVF9e/fH7YTjxDH8tSNiopSI8E6vnEPjRBSL6ReUIcg5BHPUX5/SYsWLSwWC/p70HA5j6XNP0rthGEYHeuiKIqi6A5OL/+xoCiMsTscpPyHU1FRai08z5f5UCEIId5NW6bNP0qtxe3DKL+d8dyTYRiGZThU4WEAKioK5bqALEkFp1fS5h+FchMqGg5CRUWh+BkqKgrFz1BRUSh+hoqKQvEzVFQUip+hoqJQ/AwVFYXiZ6ioKDUSyExWPaGiolD8DBUVpdJhWZb4I5kmlAOB57ClogFEtwYa+0epdCRJysvLg1mAXuSmdCPLsqqqDofDPT8XVULuW9+hoqJUOiEhIVu2bAkNDbXZbBVNTOmJKIqQWMZisVRPGwUw1blylNqBLMvuueuw7kZF8Zy3q2s6JthzilR1s1RUVJRbgReT0m9aYHXTkhvqqKDUSKqtohAVFYXid6ioKBQ/Q/tUFIqfoZaKQvn/9u41Jq5q7QP4vqy95wYzlOHSdJihXDpBCr5c3lps0VhsbNpySUwJNVo1wcsHk7aYGI1R01Rjk5rUmkabpvYQK8YeWiWkgAUr0sRiWmoLRYGWoWO51LGAMAzMzL6tdT6sOk4Re6rd5+jr+/w+EAJ7Nmtg/rPWetbabJ1BqADQGYQKAJ1BqADQGYQKAJ1BqADQGYQKAJ1BqADQGYQKAJ1BqADQGYQKAJ1BqADQGYQKAJ1BqADQGYQKAJ1BqADQGYQKAJ1BqADQGYQKAJ1BqADQGYQKAJ1BqADQGYRKB/Bv3kA0CNUCgsFg5PNIYAghC4YnckMXKhAIRH9XURRFUf6TjQV/OfDPNOej2aC3bKFfobeZ4Hk+8kV6TPRdYaKzJ0mSyWSKfEtTNY7n/sr/+xvoC0K1AEVRBEGQJGliYmJsbIzjuMTERKfTyTBMdNii/fquFjSZsizfyW3OwP9FEKqbYIxp/9PU1NTQ0DA0NMQwjCRJCQkJGRkZzz33XEZGBj2gurp6aGiouLj4xRdfjImJiZwhFAphjHft2vXVV1/l5OS89dZbZrP5z3o64E8Bc6pfEEIIJuFwuLm5+d133x0YGJBlmWVZenfN8+fP79ix49q1a4SQmZkZURQxxhhjURQjw0VCiNlsFkWR3jKQZVmawMg7V+SWZ7Is32arQqFQ5IGEEE3VIidUFIV+IknSvJupYYwVRaEtlCQp+rGKooTDYYyxLMsL3oIN3mfvENye9CaYYIZhPv/886mpqZSUlG3btq1evXpmZqa+vv6TTz7xer3Hjx9/8sknFUVBCPE8jxASBCH6DPT1vXXr1vHxcVEUDQYDTSY9LHIvQJ7j/31jMOY4zmQyBYNB2t2xLMsjHmPMsmw4HDYajfRIg8GgKIqmaYIg0ITTMNOPoihqqsYjnuVYTdUEQaCNiYxLI+dnGIYQoqoqwzDznhe4fRCqm3AspzGaLMtmszklJaWwsJBl2bi4uI0bN3Z2dsqy/O233xJCTCaTqqqapqmqqigK4lFTS9OFCxcQQiUlJbm5uRcvXhwYGEhNTc37nzxN0xoaGjweT3Z2tsPhaGxsnJiYiI2NraqqWvG/K3i0cLpoT1JbW3vu3DlFUVatWnXvvfe2trYihMrKypYtW3bmzJmOjo6UlBS32/3pp59KklRZWblmzZru7u6Wlpb+/v7p6en4+Pi8vLyqqiqHw0EIaWtru3jxYmpqamZm5pEjR7xer8vlKisrKywsPHbs2KlTp3766afs7OyqqqrMzMz/8m/+7wRCtYD4+HhN0/r7+99+++0NGza4XC6Hw1FbW4s1zCOe1s1ZltU0jY6UjvzzyEcffRQKhUpLSwsKCgghvb29HR0d99xzz8MPP8zz/Pnz5zs7OwcHB6empubm5gKBgCiKAwMDW7duXb9+/YJt8Pl8u3fv7unp4ThOVdVLly598cUXc3NzDMPcfffd6Wnp3333XUdHh81mkyRJluVQKLRx48ahoaFdu3YNDw8bDAaO4wKBwOjoqNfrffPNN3me7+npaWtrM5vNCKGpqSlJksbHxz0ej8vlGhkZmZycRAh9//33Pp/vjTfesNls/9Vf+t8IzKluwnKswWCoqqqKj4+fnZ09fvz49u3bH3nkkZqamrq6usBsgGEYrOFIiQ9jfPLkyY8//jgcDufn52/ZsoXneULI7OwsIYQuUqmqqqqqwWAYGRlZsWLF/v37X3/9dYfDoapqXV3d+Pj4vKUwQkgwGPzss88GBgY4jisrKzt48OALL7wwMzMjSZKmaWazWVEVg8GAMfb7/QzDZGVlFRQUZGRkNDQ0TE1Nud3uvXv3Hj58uKKiAmPs9Xr7+vqMRiNCCCHk8/ncbvc777zz2muv8Tzv9/t7e3srKyv379//4IMPIoS8Xu+VK1foXA78AdBT3RA9O1++fPmrr75aW1t74cIFVVVFUfzmm2+6urpaW1tfeumlnJwc2kexLPvjjz/+49A/Jicni4uLa2pqkpKSMMYLTvRZlrXb7U899ZTD4cjKypqdnT148ODMzEx/f//9999Pj1FVlU6KgsHg2bNnA4HAihUrKisrlyxZkpGeMT4+/uGHH8bGxkbOqSiKwWCorKx84vEn6DAyPT198+bNgiAkJyePjo4mJCREt0FVVUmSli5d+thjj+Xm5iYmJi5ZsmRsbCwvL2/dunUmk2nlypVffvmlpmmaptF3h+j26/sL/xuDUN0k8tLJzc3dt2+fx+Pp6+vr7u7u6+sbHR29du3aoUOHXn75ZY7jEEIcx/3www9zc3N0Uctqtd6ibhYOh5cuXRoTE0OLHC6Xi87H/H6/JEkNnzZc6L7A87ymaSaTqaSkhB4WFxfncDhkWUYIpaenRyeKYRhBEBITE/Py8lRNpeNSQRB8Pl9bW1t3d/fIyIjBYIg+3mg0hkIhWZbj4uJkWSaEiKLIcVxcXFxiYiKtfCCEaNcqCAIE6Y+BUP2C1sQZhqGzl5ycnMzMzMzMzPLy8kAgsGfPnvb2do/H4/P5nE4nLY5NT09brVZBEC5fvnzixIlNmzbRM3AcN69aTTu32NhYjuMIIQaDgVbq52bnBEG4dPnSqVOnTCZTKBSy2WwFBQUMwxBCaGU8UrUPhUIWi4X7Ge157HY74hGt7x0+fPjQoUOEEISQ0+lMTk7u7e2NtMFkMlkslpiYGKPRiDHmeZ5hmFAoREuUAhLC4TDHcXTUCon6wyBUN9BEEULOnDmzZ88eWZY3b96ctjQNEywIgtFotNvtmqbJsjwzM0Nr1sFg0Ol0btu2rb29/fTp0ydOnCguLl68eDHLsr9e/zGbzdevX/f7/Waz2WAwTE1NybIcGxvrdDkJJoWFhaIomkymubk5jHFWVhbP8zzPj46OhsNh2mmMj4/TTygaTlEUBUGgY7+JiYmmpiZFUe67777nn38+OTn566+/fuWVV2glg9YtwuEwbZ7RaBQEgZ5E0zSGYXjEGwwGeuYF16/AbYJQ/YIW97Kzs5OSkoaHhxsbGy0WS3l5+ezsbHd3d2trKyEkISHB5XIZjcZwOGyxWNxud1FRkdVq7erqGhwcbG5urq6uRjzCGCOE6NornaJwHHf9+vWmpqYtW7YEAoG2tjZJkhwOR2Jiooa10tLSdevWRZaGMMbFxcVXrlzxer319fUVFRWDg4P19fV0TEhPqKoqLbsrikJXdaenp2lUkpKS4uPjQ6HQuXPnNE0TRVFRFLoE/G/RPY2w/nsnIFQ3EEJYjmUYxmazbdq06f333/f5fO+9996BAwfoVkBZlnmeLykpcTqd09PTdE7FMAzGuKCg4IEHHjh27NjJkydXrVqVmZnJcRzP83SQRvscOlo7cOBAS0sLx3Eej8disRQVFbndbtotcOyNs9FHrVmzpr29fXh4+IMPPqirqyOELFq06Lcaz3IswSQpKclqtaqqevTo0cuXLweDwZGREbrrYnp6et78av4ZYLCnHyip3xCZTmiqtmH9hu3bt6elpWGM6WUgLMsmJCTU1NQ8+uij9L2cVrd5nqfzk7Vr16ampvp8vtOnT9NZEI0iHWLR2X92dvZDDz3k9/sHBgZsNltFRcXjjz9Ofy4hhEe8pmp0NwMhxOVy7dy5s6ioyGazcRxXWlpaXV19i/bziLeYLU8//fTy5csJIR6Px+/3V1dXp6Wlqap69epV2pv9FhppoAvYULsAOrnSNM3j8dD1VpvN5nK5EI9+10UctLoQDAZ37tzZ1dWVn5+/e/fuq1ev+v1+u92+ePHiyK6iX/8Venp6fD5ffHx8RkZGbGysKIqNjY179+5FCO3bty8rK4uGNvoH0Y+BQKCvr89qtdrtdqvVGtnKxNy8bDDvWUR2EoM7B8O/BbAsizUsCMJdd93FMAwhJBwOi6L4e8dItDCtKMrk5GQwGKRdXGpqKsEkEs4FE8UwzNmzZ48ePcrz/Nq1a5999tmxsbHOzk6GYRYtWmS32zVVo7W7eWiBceXKlczPl4FFp+UW7Yfhn44gVAvAGNN9q/SiXVEUjUZj9OWJzM+vwlu/FhFCDMPQhaDIviGzyRy93++3RgqrV69ubm72+/0tLS0NDQ30bKqqrl+/Pi4ubt6OwejGaKqmairLsgghgbvdTbEQKh3xO3bs+LPb8JdDF1vp65LW8WgZ+tfDp1u/Fumj6BqxKIputzs/P18QBLrR6dZnSEpKWrZsmSRJwWBQFEVRFB0OxzPPPFNeXn7rkgOdwtHrlP/Ikwd3DOZUt+t39VER9DINeH3/vwKh+h1gLxy4HVDwAUBnECoAdAbDPwB0Bj0VADqDUAGgMwgVADqDUAGgMwgVADqDUAGgs38BNO+5OpwvofIAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "id": "a0d649e9",
   "metadata": {},
   "source": [
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93dee758",
   "metadata": {},
   "source": [
    "## 第5步：训练Skip-Gram模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "311ccd16",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "Dimension out of range (expected to be in range of [-1, 0], but got 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Input \u001b[0;32mIn [5]\u001b[0m, in \u001b[0;36m<cell line: 10>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m y_true \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mtensor([word_to_idx[context]], dtype\u001b[38;5;241m=\u001b[39mtorch\u001b[38;5;241m.\u001b[39mlong) \u001b[38;5;66;03m# 将目标词转换为索引值  \u001b[39;00m\n\u001b[1;32m     15\u001b[0m y_pred \u001b[38;5;241m=\u001b[39m skipgram_model(X)  \u001b[38;5;66;03m# 计算预测值\u001b[39;00m\n\u001b[0;32m---> 16\u001b[0m loss \u001b[38;5;241m=\u001b[39m \u001b[43mcriterion\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_pred\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_true\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# 计算损失\u001b[39;00m\n\u001b[1;32m     17\u001b[0m loss_sum \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m loss\u001b[38;5;241m.\u001b[39mitem() \u001b[38;5;66;03m# 累积损失\u001b[39;00m\n\u001b[1;32m     18\u001b[0m optimizer\u001b[38;5;241m.\u001b[39mzero_grad()  \u001b[38;5;66;03m# 清空梯度\u001b[39;00m\n",
      "File \u001b[0;32m~/ENTER/lib/python3.9/site-packages/torch/nn/modules/module.py:1102\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1098\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1099\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1100\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1101\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1102\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1103\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1104\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/ENTER/lib/python3.9/site-packages/torch/nn/modules/loss.py:1150\u001b[0m, in \u001b[0;36mCrossEntropyLoss.forward\u001b[0;34m(self, input, target)\u001b[0m\n\u001b[1;32m   1149\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor, target: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m-> 1150\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcross_entropy\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1151\u001b[0m \u001b[43m                           \u001b[49m\u001b[43mignore_index\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mignore_index\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mreduction\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreduction\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1152\u001b[0m \u001b[43m                           \u001b[49m\u001b[43mlabel_smoothing\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlabel_smoothing\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/ENTER/lib/python3.9/site-packages/torch/nn/functional.py:2846\u001b[0m, in \u001b[0;36mcross_entropy\u001b[0;34m(input, target, weight, size_average, ignore_index, reduce, reduction, label_smoothing)\u001b[0m\n\u001b[1;32m   2844\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m size_average \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m reduce \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   2845\u001b[0m     reduction \u001b[38;5;241m=\u001b[39m _Reduction\u001b[38;5;241m.\u001b[39mlegacy_get_string(size_average, reduce)\n\u001b[0;32m-> 2846\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_C\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_nn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcross_entropy_loss\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_Reduction\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_enum\u001b[49m\u001b[43m(\u001b[49m\u001b[43mreduction\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mignore_index\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlabel_smoothing\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mIndexError\u001b[0m: Dimension out of range (expected to be in range of [-1, 0], but got 1)"
     ]
    }
   ],
   "source": [
    "# 训练Skip-Gram模型\n",
    "learning_rate = 0.001 # 设置学习速率\n",
    "epochs = 1000 # 设置训练轮次\n",
    "criterion = nn.CrossEntropyLoss()  # 定义交叉熵损失函数\n",
    "import torch.optim as optim # 导入随机梯度下降优化器\n",
    "optimizer = optim.SGD(skipgram_model.parameters(), lr=learning_rate)  \n",
    "\n",
    "# 开始训练循环\n",
    "loss_values = []  # 用于存储每轮的平均损失值\n",
    "for epoch in range(epochs):\n",
    "    loss_sum = 0 # 初始化损失值\n",
    "    for context, target in skipgram_data:        \n",
    "        X = one_hot_encoding(target, word_to_idx).float().unsqueeze(0) # 将中心词转换为One-Hot向量   \n",
    "        y_true = torch.tensor([word_to_idx[context]], dtype=torch.long) # 将周围词转换为索引值   \n",
    "        y_pred = skipgram_model(X)  # 计算预测值\n",
    "        loss = criterion(y_pred, y_true)  # 计算损失\n",
    "        loss_sum += loss.item() # 累积损失\n",
    "        optimizer.zero_grad()  # 清空梯度\n",
    "        loss.backward()  # 反向传播\n",
    "        optimizer.step()  # 更新参数\n",
    "    if (epoch+1) % 100 == 0: # 输出每100轮的损失，并记录损失\n",
    "        print(f\"Epoch: {epoch+1}, Loss: {loss_sum/len(skipgram_data)}\")  \n",
    "        loss_values.append(loss_sum / len(skipgram_data))\n",
    "\n",
    "# 绘制训练损失曲线\n",
    "import matplotlib.pyplot as plt # 导入matplotlib\n",
    "plt.plot(range(1, epochs//100 + 1), loss_values) # 绘图\n",
    "plt.title('Training Loss') # 图题\n",
    "plt.xlabel('Epochs') # X轴Label\n",
    "plt.ylabel('Loss') # Y轴Label\n",
    "plt.show() # 显示图"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca992d08",
   "metadata": {},
   "source": [
    "## 第6步：输出Skip-Gram词向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75e56ed1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 输出Skip-Gram习得的词嵌入\n",
    "print(\"\\nSkip-Gram词嵌入:\")\n",
    "for word, idx in word_to_idx.items(): # 输出每个单词的嵌入向量\n",
    "    print(f\"{word}: \\\n",
    "    {skipgram_model.input_to_hidden.weight[:, idx].detach().numpy()}\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1670078",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"font.family\"]=['SimHei'] # 用来设定字体样式\n",
    "plt.rcParams['font.sans-serif']=['SimHei'] # 用来设定无衬线字体样式\n",
    "plt.rcParams['axes.unicode_minus']=False # 用来正常显示负号\n",
    "# 绘制二维词向量图\n",
    "fig, ax = plt.subplots() \n",
    "for word, idx in word_to_idx.items():\n",
    "    vec = skipgram_model.input_to_hidden.weight[:, \\\n",
    "            idx].detach().numpy() # 获取每个单词的嵌入向量\n",
    "    ax.scatter(vec[0], vec[1]) # 在图中绘制嵌入向量的点\n",
    "    ax.annotate(word, (vec[0], vec[1]), fontsize=12) # 点旁添加单词标签\n",
    "plt.title('2维词嵌入') # 图题\n",
    "plt.xlabel('向量维度1') # X轴Label\n",
    "plt.ylabel('向量维度2') # Y轴Label\n",
    "plt.show() # 显示图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27c673b0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
